# -*- coding: utf-8 -*-
from scrapy.spider import CrawlSpider, Rule
from scrapy.linkextractors import LinkExtractor
from scrapy.http import TextResponse
from baike_avp.items import BaikeAvpItem
from ..middlewares import redis_db

import urllib2
import re
import random
import codecs


class ArticleSpider(CrawlSpider):
    """
    根据start_urls定义范围内所有的词条页面，及其子页面，抓取xpath抽取到的元素。
    一定要使用重载的parse_start_url作为回调函数，否则爬虫将无法爬取start_urls
    页面，只能爬取其子页面（根据rules抽取到的页面）。
    """

    name = 'baidu_avp'
    allowed_domains = ['baike.baidu.com']

    # start_urls = ['http://baike.baidu.com/view/' + str(pageId) + '.htm' for pageId in xrange(0, 15000000)]
    # random.shuffle(start_urls)

    start_urls = [kv[0] for kv in redis_db.hgetall('baidu_req_url').items() if kv[1] == '0']

    # For Debug
    # start_urls = ['https://baike.baidu.com/item/同济大学']

    rules = (
        Rule(LinkExtractor(allow=(r'/item/.*',)), follow=True, callback='parse_start_url'),
    )

    # object纯文本中的特殊字符：顿号、正斜线、反斜线、逗号
    special_chars = (u'\u3001', '\\', '/', u'\uff0c', ',')

    # 信息盒模式
    infobox_pattern = '//*[@class="basic-info cmn-clearfix"]'
    # 词条标题模式，用于subject
    title_pattern = '//*[@class="lemmaWgt-lemmaTitle-title"]/h1/text()'
    # 副标题模式，用于tag
    subtitle_pattern = '//*[@class="lemmaWgt-lemmaTitle-title"]/h2/text()'
    # 词条第一段描述模式，用于desc
    desc_pattern = '//*[@class="lemma-summary"]/div[@class="para"]'
    # 谓语模式，即信息盒中的属性名字
    predicates_pattern = '//*[@class="basic-info cmn-clearfix"]/dl//*[@class="basicInfo-item name"]/text()'
    # 客体模式，即信息盒中的属性值
    objects_pattern = '//*[@class="basic-info cmn-clearfix"]/dl//*[@class="basicInfo-item value"]'
    # 词条标签模式，用于label
    labels_pattern = '//*[@class="taglist"]/text()'
    # 词条标签中的超链接模式，用于label
    hyper_label_pattern = '//*[@class="taglist"]/a/text()'
    # 词条最近更新模式
    # modified_time_pattern = '//*[@class="j-modified-time"]/text()'
    # 词条主要内容模式
    main_content_pattern = '//div[@class="main-content"]'

    # 用户代理列表
    user_agent_list = [
        "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1",
        "Mozilla/5.0 (X11; CrOS i686 2268.111.0) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.57 Safari/536.11",
        "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1092.0 Safari/536.6",
        "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1090.0 Safari/536.6",
        "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/19.77.34.5 Safari/537.1",
        "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.9 Safari/536.5",
        "Mozilla/5.0 (Windows NT 6.0) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.36 Safari/536.5",
        "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3",
        "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3",
        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_0) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3",
        "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3",
        "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3",
        "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3",
        "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3",
        "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3",
        "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.0 Safari/536.3",
        "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24",
        "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24"
    ]

    @staticmethod
    def title_rule(title):
        # 用下划线替换空格
        title = title.replace(' ', '_')

        return title

    @staticmethod
    def subtitle_rule(subtitle):
        # 用下划线替换空格
        subtitle = subtitle.replace(' ', '_')
        # 用斜线替换反斜线
        subtitle = subtitle.replace('\\', '/')
        # 用单引号替换双引号
        subtitle = subtitle.replace('"', "'")

        return subtitle

    @staticmethod
    def predicate_rule(predicate):
        # 清空无效字符
        predicate = predicate.replace(u'\xa0', '').replace(' ', '').replace('"', '').replace('\t', '')
        # 用斜线替换反斜线
        predicate = predicate.replace('\\', '/')

        return predicate

    @staticmethod
    def attribute_rule(attribute):
        # 清空无效字符
        attribute = attribute.replace('\n', '').replace('\r', '').replace(u'\u3000', '')
        # 用下划线替换空格
        attribute = attribute.replace(u'\xa0', '_').replace(' ', '_')
        # 用单引号替换双引号
        attribute = attribute.replace('"', "'")

        return attribute

    @staticmethod
    def desc_rule(description):
        # 清空无效字符
        description = description.replace('\n', '').replace('\r', '').replace(u'\u3000', '')
        # 用下划线替换空格
        description = description.replace(u'\xa0', '_').replace(' ', '_')
        # 用斜线替换反斜线
        description = description.replace('\\', '/')
        # 用单引号替换双引号
        description = description.replace('"', "'")

        return description

    def get_hyper_title_tag(self, title_url):
        ua = random.choice(self.user_agent_list)
        full_url = 'https://baike.baidu.com' + title_url
        request = urllib2.Request(full_url)
        request.add_header("User-Agent", ua)
        while True:
            try:
                response = urllib2.urlopen(request, timeout=5)
                res_data = response.read()
            except:
                print u'Page loading timeout, reload now.'
                with codecs.open('timeout_url.txt', mode='a') as timeout_url:
                    timeout_url.write(full_url + '\n')
                break
            else:
                break
        res = TextResponse(url=full_url, body=res_data)
        raw_hyper_title = res.xpath(self.title_pattern).extract()[0]
        raw_hyper_tag = ''.join(res.xpath(self.subtitle_pattern).extract())

        return raw_hyper_title, raw_hyper_tag

    # 检查链接所指内容是否确实与当前实体有关
    def check_relation(self, entry_url, title):
        ua = random.choice(self.user_agent_list)
        full_url = 'https://baike.baidu.com' + entry_url
        request = urllib2.Request(full_url)
        request.add_header("User-Agent", ua)
        while True:
            try:
                response = urllib2.urlopen(request, timeout=3)
                res_data = response.read()
            except:
                print u'Page loading timeout, reload now.'
            else:
                break
        res = TextResponse(url=full_url, body=res_data)
        main_content = res.xpath(self.main_content_pattern).extract()[0]
        if title in main_content:
            return True
        else:
            return False

    def parse_start_url(self, response):

        # 判断词条页面是否有InfoBox
        if response.xpath(self.infobox_pattern):
            item = BaikeAvpItem()

            raw_labels = response.xpath(self.labels_pattern).extract() + response.xpath(self.hyper_label_pattern).extract()
            context_list = [tag.strip() for tag in raw_labels if tag != '\n']
            raw_subtitle = ''.join(response.xpath(self.subtitle_pattern).extract())
            modified_subtitle = self.subtitle_rule(raw_subtitle[1:-1])
            if modified_subtitle:
                item['tag'] = modified_subtitle
            else:
                item['tag'] = u'默认标签'
            if not context_list:
                context_list = [u'无类别']
            # count = len(context_list)

            raw_title = response.xpath(self.title_pattern).extract()[0]
            modified_title = self.title_rule(raw_title)
            item['subject'] = modified_title

            try:
                raw_desc = response.xpath(self.desc_pattern).extract_first()
            except TypeError:
                raw_desc = u'暂无描述'
            re_desc = re.compile(r'<.*?>')
            modified_desc = re.sub(re_desc, '', raw_desc)
            modified_desc = self.desc_rule(modified_desc)
            item['desc'] = modified_desc

            # item['modified_time'] = response.xpath(self.modified_time_pattern).extract()[0]
            # item['url'] = response.url

            predicates = response.xpath(self.predicates_pattern).extract()
            objects = response.xpath(self.objects_pattern)

            item['label'] = context_list

            # po_count = len(predicates)

            po_pairs = zip(predicates, objects)

            # po_pairs = zip(predicates, objects) * count

            predicate_list = []
            object_list = []
            for idx, po_pair in enumerate(po_pairs):
                # 先获取object部分的全部纯文本
                object_value = po_pair[1].xpath('text()').extract()

                # 获得关系名
                # item['predicate'] = self.predicate_rule(po_pair[0])
                # item['label'] = self.attribute_rule(context_list[idx // po_count])
                # 对于有超链接的尾实体
                if po_pair[1].xpath('a'):
                    # 仅获取单独一个超链接组成的尾实体
                    if len(object_value) == 2 and (object_value[0] == '\n' and object_value[1] == '\n' or object_value[0] == u'\n\u300a' and object_value[1] == u'\u300b\n'):
                        object_url = po_pair[1].xpath('a/@href').extract()[0]
                        hyper_object = self.get_hyper_title_tag(object_url)
                        modified_object = self.title_rule(hyper_object[0])
                        modified_tag = self.subtitle_rule(hyper_object[1]) if self.subtitle_rule(hyper_object[1]) else u'默认标签'
                        predicate_list.append(self.predicate_rule(po_pair[0]))
                        object_list.append('|||' + modified_object + '|||' + modified_tag)
                        # item['object_entity'] = modified_object
                        # item['object_literal'] = ''
                        # yield item
                    else:
                        continue
                # 纯文本的尾实体
                else:
                    if len(object_value) > 1 or any(special_char in object_value[0] for special_char in self.special_chars):
                        continue
                    else:
                        predicate_list.append(self.predicate_rule(po_pair[0]))
                        object_list.append(self.attribute_rule(object_value[0]))
                        # item['object_literal'] = self.attribute_rule(object_value[0])
                        # item['object_entity'] = ''
                        # yield item
            assert len(predicate_list) == len(object_list)
            item['predicate'] = predicate_list
            item['object_value'] = object_list
            yield item
